import LConf
import random
import os
import requests
from pyquery import PyQuery as pq
from EasySqlite import EasySqlite

class Spider:
    # uA池
    UaPool = []
    # 配置文件内容
    conf = None
    # 配置情况
    iniFile = 'spi_txt.ini'

    def __init__(self):
        # 当前脚本根目录
        self._path = os.path.split(os.path.realpath(__file__))[0]
        # 加载配置文件内容
        self.conf = LConf.iniConf(self._path + '/' + self.iniFile)
        # 加载所有UA库
        self.UaPool = LConf.rJson(self._path + '/' + self.conf['ua'])
        dbFile = self._path + '/'+ self.conf['resPath']
        if (os.path.isfile(dbFile)):
            self.db = EasySqlite(dbFile)
        else:
            self.db = EasySqlite(dbFile)
            self.db.execSqlScript(self._path + '/' + self.conf['initSql'])

    # 随机获取一个ua
    def getUa(self):
        r = random.randint(0, len(self.UaPool) - 1)
        return {'user-agent': self.UaPool[r]}

    # 完成一个任务
    def getOne(self, url):
        h = self.getUa()
        r = requests.get(url, headers=h, timeout=int(self.conf['timeout']))
        r.encoding = 'UTF-8'
        html = r.text
        dom = pq(html)
        titleDom = dom('div.news_details')
        title = titleDom('h1').text()
        contentDom = dom('div.xs-details-content')
        content = contentDom.text()
        return (content,title)

    # 获取新任务（添加任务池）
    def getList(self, url):
        urlBase = url.split('?')[0]
        retItems = []
        h = self.getUa()
        r = requests.get(url, headers=h, timeout=int(self.conf['timeout']))
        r.encoding = 'UTF-8'
        html = r.text
        dom = pq(html)
        classDom = dom('div.box-title h3')
        classType = classDom.text()
        ulDom = dom('div.layout-box ul')
        ulDom = ulDom.eq(1)
        aDoms = ulDom('a')
        np = dom('#long-page a.next')
        npUrl = urlBase + np.attr.href
        for item in aDoms.items():
            retItems.append(self.conf['domain'] + item.attr.href)
        return (retItems, npUrl, classType)

    # 初始化任务
    def iniTask(self):
        start_urls = self.conf['start_url']
        tasks = []
        for one in start_urls:
            tasks.append(self.conf['domain'] + one)
        self.addTasks(tasks, 1)

    # 从任务池获取一条任务
    def getTask(self):
        # 优先获取正常的任务
        item = self.db.table('task').where(['status=0']).find()
        if item:
            return {'id': item[0], 'url': item[1],'type': item[4], 'is_add': item[5]}
        # 所有正常任务完成，再获取失败的任务
        item = self.db.table('task').where(['status=-1','re_time<5']).find()
        if item:
            return {'id': item[0], 'url': item[1],'type': item[4], 'is_add': item[5]}
        return False


    def main(self):
        # 初始化任务
        self.iniTask()
        # 任务循环
            # 从任务池中取一个任务
        while 1:
            aTask = self.getTask()
            taskCount = self.db.table('task').where('status=0 OR (status=-1 and re_time<5)').count()
            if aTask is False:
                return 0
            taskSuss = False
            printStr = ''
            if aTask['is_add'] == 1:
                try:
                    tasks, np, classType = self.getList(aTask['url'])
                    self.addTasks(tasks, 0, classType)
                    self.addTasks(np, 1)
                    printStr += '【成功】新增任务 +%d (%d/%d)' % (len(tasks), aTask['id'], taskCount)
                    # 记录成功
                    taskSuss = True
                except Exception as e:
                    # 记录失败
                    theClass = False
            else:
                try:
                    content, title = self.getOne(aTask['url'])
                    self.saveRes(aTask['url'], content, title, aTask['type'])
                    taskSuss = True
                    if '0' == self.conf['print_title']:
                        title = "******"
                    printStr += '【成功】 %s (%d/%d)' % (title, aTask['id'], taskCount)
                except Exception as e:
                    taskSuss = False
            if taskSuss:
                self.db.table('task').where('id=%d' % aTask['id']).update({'status': 4})
            else:
                printStr += '【失败】任务号%d (%d/%d)' %(aTask['id'], aTask['id'], taskCount)
                self.db.table('task').where('id=%d' % aTask['id']).update({'status': -1})
            LConf.Wlog(self._path + '/' + self.conf['logPath'], printStr, 'a')
            print(printStr)
            self.db.commit()

    def addTasks(self, urls, ty, classType = ''):
        if isinstance(urls, str):
            urls = (urls,)
        for url in urls:
            self.db.table('task').insert({'url': url, 'is_add': ty, 'type': classType})
        self.db.commit()

    def saveRes(self, url, content, title, ty):
        self.db.table('res').insert({'url': url, 'content': content, 'title': title, 'type':ty})
        self.db.commit()

if __name__ == "__main__":
    spi = Spider()
    spi.main()
    # tasks, np, tp = spi.getList('http://www.aah49.com/novellist.x?classid=1')
    # spi.addTasks(tasks, 0, tp)
    # spi.addTasks(np,1)

    # res = spi.getTask()
    # spi.iniTask()
    
    # url = 'http://www.aah49.com/pn.x?stype=novel&id=127556'
    # content, title = spi.getOne(url)
    # classType = "***"
    # spi.saveRes(url, content, title, classType)

